package com.doit.day04

import com.alibaba.fastjson.JSON
import org.apache.spark.{SparkConf, SparkContext}

/**
 * @Author:
 * @WX: 17710299606
 * @Tips: 学大数据 ,到多易教育
 * @DOC: https://blog.csdn.net/qq_37933018?spm=1000.2115.3001.5343
 * @Description:
 */
object Demo02ParseJson {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
      .setMaster("local[*]")
      .setAppName(this.getClass.getSimpleName)
    val sc = SparkContext.getOrCreate(conf)


    var cnt: Int = 0
    // 使用全局计数器
    val cnt1 = sc.longAccumulator("cnt")


    sc.textFile("data/log/")
      .map(line => {
        var logBean = new LogBean(-1, "", "", -1L)
        try {
          logBean = JSON.parseObject(line, classOf[LogBean])
          logBean
        } catch {
          case e: Exception => {
            cnt += 1
            cnt1.add(1)
            logBean
          }
        }
      }).collect()
    // 获取全局计数结果
    println(cnt1.value)
    println(cnt)


  }

}
